<!DOCTYPE html>
<html lang="zh">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.3.0">


  <link rel="apple-touch-icon" sizes="180x180" href="/yuwanzi.io/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/yuwanzi.io/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/yuwanzi.io/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/yuwanzi.io/images/logo.svg" color="#222">

<link rel="stylesheet" href="/yuwanzi.io/css/main.css">



<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.1/css/all.min.css">
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/animate.css@3.1.1/animate.min.css">

<script class="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"suyuhuan.gitee.io","root":"/yuwanzi.io/","images":"/yuwanzi.io/images","scheme":"Muse","version":"8.2.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":false,"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"Suche...","empty":"We didn't find any results for the search: ${query}","hits_time":"${hits} results found in ${time} ms","hits":"${hits} results found"}};
  </script>
<meta name="description" content="什么是爬虫? 如果是没有接触过爬虫的人可能会有些许疑惑，爬虫是个什么东西呢？其实爬虫的概念很简单，在互联网时代,万维网已然是大量信息的载体，如何有效地利用并提取这些信息是一个巨大的挑战。当我们使用浏览器对某个网站发送请求时，服务器会响应HTML文本并由浏览器来进行渲染显示。爬虫正是利用了这一点，通过程序模拟用户的请求，来获得HTML的内容，并从中提取需要的数据和信息。如果把网络想象成一张蜘蛛网，爬">
<meta property="og:type" content="article">
<meta property="og:title" content="教你如何快速实现一个图片爬虫">
<meta property="og:url" content="https://suyuhuan.gitee.io/yuwanzi.io/2017/09/20/2017-09-20-PictureSpider/index.html">
<meta property="og:site_name" content="玉丸子 | Blog">
<meta property="og:description" content="什么是爬虫? 如果是没有接触过爬虫的人可能会有些许疑惑，爬虫是个什么东西呢？其实爬虫的概念很简单，在互联网时代,万维网已然是大量信息的载体，如何有效地利用并提取这些信息是一个巨大的挑战。当我们使用浏览器对某个网站发送请求时，服务器会响应HTML文本并由浏览器来进行渲染显示。爬虫正是利用了这一点，通过程序模拟用户的请求，来获得HTML的内容，并从中提取需要的数据和信息。如果把网络想象成一张蜘蛛网，爬">
<meta property="og:locale">
<meta property="og:image" content="http://wx1.sinaimg.cn/large/63503acbly1fjrjc53yfgj20ze0hj4fa.jpg">
<meta property="og:image" content="http://wx3.sinaimg.cn/large/63503acbly1fjrjc67dulj21eg0h2qpr.jpg">
<meta property="og:image" content="http://wx1.sinaimg.cn/large/63503acbly1fjrjc5msszj20sn0fkn80.jpg">
<meta property="og:image" content="http://wx2.sinaimg.cn/large/63503acbly1fjrmgmpigfg214o0ihe2b.gif">
<meta property="article:published_time" content="2017-09-20T10:00:00.000Z">
<meta property="article:modified_time" content="2020-11-07T00:58:17.000Z">
<meta property="article:author" content="玉丸子">
<meta property="article:tag" content="2017">
<meta property="article:tag" content="python">
<meta property="article:tag" content="爬虫">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://wx1.sinaimg.cn/large/63503acbly1fjrjc53yfgj20ze0hj4fa.jpg">


<link rel="canonical" href="https://suyuhuan.gitee.io/yuwanzi.io/2017/09/20/2017-09-20-PictureSpider/">


<script class="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh'
  };
</script>
<title>教你如何快速实现一个图片爬虫 | 玉丸子 | Blog</title>
  




  <noscript>
  <style>
  body { margin-top: 2rem; }

  .use-motion .menu-item,
  .use-motion .sidebar,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header {
    visibility: visible;
  }

  .use-motion .header,
  .use-motion .site-brand-container .toggle,
  .use-motion .footer { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle,
  .use-motion .custom-logo-image {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line {
    transform: scaleX(1);
  }

  .search-pop-overlay, .sidebar-nav { display: none; }
  .sidebar-panel { display: block; }
  </style>
</noscript>

<link rel="alternate" href="/yuwanzi.io/atom.xml" title="玉丸子 | Blog" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Navigationsleiste an/ausschalten" role="button">
    </div>
  </div>

  <div class="site-meta">

    <a href="/yuwanzi.io/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">玉丸子 | Blog</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>







</div>
        
  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>

  <aside class="sidebar">

    <div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          Inhaltsverzeichnis
        </li>
        <li class="sidebar-nav-overview">
          Übersicht
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
            <div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BB%80%E4%B9%88%E6%98%AF%E7%88%AC%E8%99%AB"><span class="nav-number">1.</span> <span class="nav-text">什么是爬虫?</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#BeautifulSoup"><span class="nav-number">2.</span> <span class="nav-text">BeautifulSoup</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#Scrapy"><span class="nav-number">3.</span> <span class="nav-text">Scrapy</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#items"><span class="nav-number">3.1.</span> <span class="nav-text">items</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#pipelines"><span class="nav-number">3.2.</span> <span class="nav-text">pipelines</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#spiders"><span class="nav-number">3.3.</span> <span class="nav-text">spiders</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%85%B6%E4%BB%96%E4%BE%9D%E8%B5%96%E5%BA%93"><span class="nav-number">4.</span> <span class="nav-text">其他依赖库</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Requests"><span class="nav-number">4.1.</span> <span class="nav-text">Requests</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#BloomFilter"><span class="nav-number">4.2.</span> <span class="nav-text">BloomFilter</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%88%86%E6%9E%90"><span class="nav-number">5.</span> <span class="nav-text">分析</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%88%9D%E5%A7%8B%E9%85%8D%E7%BD%AE"><span class="nav-number">6.</span> <span class="nav-text">初始配置</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E8%A7%A3%E6%9E%90%E7%83%AD%E9%97%A8%E9%A1%B5%E9%9D%A2"><span class="nav-number">7.</span> <span class="nav-text">解析热门页面</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E8%A7%A3%E6%9E%90%E8%AF%A6%E6%83%85%E9%A1%B5"><span class="nav-number">8.</span> <span class="nav-text">解析详情页</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%A4%84%E7%90%86Item"><span class="nav-number">9.</span> <span class="nav-text">处理Item</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#IP%E4%BB%A3%E7%90%86%E6%B1%A0"><span class="nav-number">10.</span> <span class="nav-text">IP代理池</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#End"><span class="nav-number">11.</span> <span class="nav-text">End</span></a></li></ol></div>
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author site-overview-item animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">玉丸子</p>
  <div class="site-description" itemprop="description">这里是玉丸子的个人博客,与你一起发现更大的世界。</div>
</div>
<div class="site-state-wrap site-overview-item animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/yuwanzi.io/archives">
          <span class="site-state-item-count">68</span>
          <span class="site-state-item-name">Artikel</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/yuwanzi.io/categories/">
        <span class="site-state-item-count">39</span>
        <span class="site-state-item-name">Kategorien</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/yuwanzi.io/tags/">
        <span class="site-state-item-count">46</span>
        <span class="site-state-item-name">schlagwörter</span></a>
      </div>
  </nav>
</div>



        </div>
      </div>
    </div>
  </aside>
  <div class="sidebar-dimmer"></div>


    </header>

    
  <div class="back-to-top" role="button">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


    <div class="main-inner post posts-expand">


  


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh">
    <link itemprop="mainEntityOfPage" href="https://suyuhuan.gitee.io/yuwanzi.io/2017/09/20/2017-09-20-PictureSpider/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/yuwanzi.io/images/avatar.gif">
      <meta itemprop="name" content="玉丸子">
      <meta itemprop="description" content="这里是玉丸子的个人博客,与你一起发现更大的世界。">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="玉丸子 | Blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          教你如何快速实现一个图片爬虫
        </h1>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">Veröffentlicht am</span>

      <time title="Erstellt: 2017-09-20 18:00:00" itemprop="dateCreated datePublished" datetime="2017-09-20T18:00:00+08:00">2017-09-20</time>
    </span>
      <span class="post-meta-item">
        <span class="post-meta-item-icon">
          <i class="far fa-calendar-check"></i>
        </span>
        <span class="post-meta-item-text">Bearbeitet am</span>
        <time title="Geändert am: 2020-11-07 08:58:17" itemprop="dateModified" datetime="2020-11-07T08:58:17+08:00">2020-11-07</time>
      </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">in</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/" itemprop="url" rel="index"><span itemprop="name">后端</span></a>
        </span>
          . 
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/python/" itemprop="url" rel="index"><span itemprop="name">python</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
        <h3 id="什么是爬虫"><a href="#什么是爬虫" class="headerlink" title="什么是爬虫?"></a>什么是爬虫?</h3><hr>
<p>如果是没有接触过爬虫的人可能会有些许疑惑，爬虫是个什么东西呢？其实爬虫的概念很简单，在互联网时代,万维网已然是大量信息的载体，如何有效地利用并提取这些信息是一个巨大的挑战。<strong>当我们使用浏览器对某个网站发送请求时，服务器会响应<code>HTML</code>文本并由浏览器来进行渲染显示。爬虫正是利用了这一点，通过程序模拟用户的请求，来获得<code>HTML</code>的内容，并从中提取需要的数据和信息</strong>。如果把网络想象成一张蜘蛛网，爬虫程序则像是蜘蛛网上的蜘蛛，不断地爬取数据与信息。</p>
<p>爬虫的概念非常简单易懂，利用<code>python</code>内置的<code>urllib</code>库都可以实现一个简单的爬虫，下面的代码是一个非常简单的爬虫，只要有基本的<code>python</code>知识应该都能看懂。它会收集一个页面中的所有<code>&lt;a&gt;</code>标签(没有做任何规则判断)中的链接，然后顺着这些链接不断地进行深度搜索。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line"></span><br><span class="line"><span class="comment"># 网页的实体类,只含有两个属性,url和标题</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Page</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self,url,title</span>):</span></span><br><span class="line">        self._url = url</span><br><span class="line">        self._title = title</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__str__</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">&#x27;[Url]: %s [Title]: %s&#x27;</span> %(self._url,self._title)</span><br><span class="line"></span><br><span class="line">    __repr__ = __str__</span><br><span class="line"></span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">url</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">return</span> self._url</span><br><span class="line"></span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">title</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">return</span> self._title</span><br><span class="line"></span><br><span class="line"><span class="meta">    @url.setter</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">url</span>(<span class="params">self,value</span>):</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="built_in">isinstance</span>(value,<span class="built_in">str</span>):</span><br><span class="line">            <span class="keyword">raise</span> ValueError(<span class="string">&#x27;url must be a string!&#x27;</span>)</span><br><span class="line">        <span class="keyword">if</span> value == <span class="string">&#x27;&#x27;</span>:</span><br><span class="line">            <span class="keyword">raise</span> ValueError(<span class="string">&#x27;url must be not empty!&#x27;</span>)</span><br><span class="line">        self._url = value</span><br><span class="line"></span><br><span class="line"><span class="meta">    @title.setter</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">title</span>(<span class="params">self,value</span>):</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="built_in">isinstance</span>(value,<span class="built_in">str</span>):</span><br><span class="line">            <span class="keyword">raise</span> ValueError(<span class="string">&#x27;title must be a string!&#x27;</span>)</span><br><span class="line">        <span class="keyword">if</span> value == <span class="string">&#x27;&#x27;</span>:</span><br><span class="line">            <span class="keyword">raise</span> ValueError(<span class="string">&#x27;title must be not empty!&#x27;</span>)</span><br><span class="line">        self._title = value</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Spider</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self,init_page</span>):</span></span><br><span class="line">        self._init_page = init_page <span class="comment"># 种子网页,也就是爬虫的入口</span></span><br><span class="line">        self._pages = []</span><br><span class="line">        self._soup = <span class="literal">None</span> <span class="comment"># BeautifulSoup 一个用来解析HTML的解析器</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">crawl</span>(<span class="params">self</span>):</span></span><br><span class="line">        start_time = datetime.now()</span><br><span class="line">        print(<span class="string">&#x27;[Start Time]: %s&#x27;</span> % start_time)</span><br><span class="line">        start_timestamp = start_time.timestamp()</span><br><span class="line">        tocrawl = [self._init_page] <span class="comment"># 记录将要爬取的网页</span></span><br><span class="line">        crawled = [] <span class="comment"># 记录已经爬取过的网页</span></span><br><span class="line">		<span class="comment"># 不断循环,直到将这张图搜索完毕</span></span><br><span class="line">        <span class="keyword">while</span> tocrawl:</span><br><span class="line">            page = tocrawl.pop()</span><br><span class="line">            <span class="keyword">if</span> page <span class="keyword">not</span> <span class="keyword">in</span> crawled:</span><br><span class="line">                self._init_soup(page)</span><br><span class="line">                self._packaging_to_pages(page)</span><br><span class="line">                links = self._extract_links()</span><br><span class="line">                self._union_list(tocrawl,links)</span><br><span class="line">                crawled.append(page)</span><br><span class="line">        self._write_to_curdir()</span><br><span class="line">        end_time = datetime.now()</span><br><span class="line">        print(<span class="string">&#x27;[End Time]: %s&#x27;</span> % end_time)</span><br><span class="line">        end_timestamp = end_time.timestamp()</span><br><span class="line">        print(<span class="string">&#x27;[Total Time Consuming]: %f.3s&#x27;</span> % (start_timestamp - end_timestamp) / <span class="number">1000</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_init_soup</span>(<span class="params">self,page</span>):</span></span><br><span class="line">        page_content = <span class="literal">None</span></span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">			<span class="comment"># urllib可以模拟用户请求,获得响应的HTML文本内容</span></span><br><span class="line">            page_content = urllib.request.urlopen(page).read()</span><br><span class="line">        <span class="keyword">except</span>:</span><br><span class="line">            page_content = <span class="string">&#x27;&#x27;</span></span><br><span class="line">		<span class="comment"># 初始化BeautifulSoup,参数二是使用到的解析器名字	</span></span><br><span class="line">        self._soup = BeautifulSoup(page_content,<span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_extract_links</span>(<span class="params">self</span>):</span></span><br><span class="line">        a_tags = self._soup.find_all(<span class="string">&#x27;a&#x27;</span>) <span class="comment"># 找到所有a标签</span></span><br><span class="line">        links = []</span><br><span class="line">		<span class="comment"># 收集所有a标签中的链接</span></span><br><span class="line">        <span class="keyword">for</span> a_tag <span class="keyword">in</span> a_tags:</span><br><span class="line">            links.append(a_tag.get(<span class="string">&#x27;href&#x27;</span>))</span><br><span class="line">        <span class="keyword">return</span> links</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_packaging_to_pages</span>(<span class="params">self,page</span>):</span></span><br><span class="line">        title_string = <span class="string">&#x27;&#x27;</span></span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            title_string = self._soup.title.string <span class="comment"># 获得title标签中的文本内容</span></span><br><span class="line">        <span class="keyword">except</span> AttributeError <span class="keyword">as</span> e :</span><br><span class="line">            print(e)</span><br><span class="line">        page_obj = Page(page,title_string)</span><br><span class="line">        print(page_obj)</span><br><span class="line">        self._pages.append(page_obj)</span><br><span class="line"></span><br><span class="line">	<span class="comment"># 将爬取到的所有信息写入到当前目录下的out.txt文件</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_write_to_curdir</span>(<span class="params">self</span>):</span></span><br><span class="line">        cur_path = os.path.join(os.path.abspath(<span class="string">&#x27;.&#x27;</span>),<span class="string">&#x27;out.txt&#x27;</span>)</span><br><span class="line">        print(<span class="string">&#x27;Start write to %s&#x27;</span> % cur_path)</span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(cur_path,<span class="string">&#x27;w&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">            f.write(self._pages)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将dest中的不存在于src的元素合并到src</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_union_list</span>(<span class="params">self,src,dest</span>):</span></span><br><span class="line">        <span class="keyword">for</span> dest_val <span class="keyword">in</span> dest:</span><br><span class="line">            <span class="keyword">if</span> dest_val <span class="keyword">not</span> <span class="keyword">in</span> src:</span><br><span class="line">                src.append(dest_val)</span><br><span class="line"></span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">init_page</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">return</span> self._init_page</span><br><span class="line"></span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pages</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">return</span> self._pages</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">test</span>():</span></span><br><span class="line">    spider = Spider(<span class="string">&#x27;https://sylvanassun.github.io/&#x27;</span>)</span><br><span class="line">    spider.crawl()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    test()</span><br></pre></td></tr></table></figure>

<p>但是我们如果想要实现一个性能高效的爬虫，那需要的复杂度也会增长，本文旨在快速实现，所以我们需要借助他人实现的爬虫框架来当做脚手架，在这之上来构建我们的图片爬虫(如果有时间的话当然也鼓励自己造轮子啦)。</p>
<blockquote>
<p>本文作者为: <a target="_blank" rel="noopener" href="https://github.com/SylvanasSun">SylvanasSun(sylvanas.sun@gmail.com)</a>.转载请务必将下面这段话置于文章开头处(保留超链接).<br>本文首发自<a target="_blank" rel="noopener" href="https://sylvanassun.github.io/">SylvanasSun Blog</a>,原文链接: <a target="_blank" rel="noopener" href="https://sylvanassun.github.io/2017/09/20/2017-09-20-PictureSpider/">https://sylvanassun.github.io/2017/09/20/2017-09-20-PictureSpider/</a></p>
</blockquote>
<h3 id="BeautifulSoup"><a href="#BeautifulSoup" class="headerlink" title="BeautifulSoup"></a>BeautifulSoup</h3><hr>
<p><strong>BeautifulSoup是一个用于从<code>HTML</code>和<code>XML</code>中提取数据的<code>python</code>库</strong>。Beautiful Soup自动将输入文档转换为Unicode编码，输出文档转换为utf-8编码。你不需要考虑编码方式，除非文档没有指定一个编码方式，这时，Beautiful Soup就不能自动识别编码方式了。然后，你仅仅需要说明一下原始编码方式就可以了。</p>
<p>利用好BeautifulSoup可以为我们省去许多编写正则表达式的时间，如果当你需要更精准地进行搜索时，BeautifulSoup也支持使用正则表达式进行查询。</p>
<p>BeautifulSoup3已经停止维护了，现在基本使用的都是BeautifulSoup4，安装BeautifulSoup4很简单，只需要执行以下的命令。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install beautifulsoup4</span><br></pre></td></tr></table></figure>
<p>然后从<code>bs4</code>模块中导入BeautifulSoup对象，并创建这个对象。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"></span><br><span class="line">soup = BeautifulSoup(body,<span class="string">&#x27;lxml&#x27;</span>)</span><br></pre></td></tr></table></figure>
<p>创建BeautifulSoup对象需要传入两个参数,第一个是需要进行解析的<code>HTML</code>内容，第二个参数为解析器的名字(如果不传入这个参数，BeautifulSoup会默认使用<code>python</code>内置的解析器<code>html.parser</code>)。BeautifulSoup支持多种解析器，有<code>lxml</code>、<code>html5lib</code>、<code>html.parser</code>。</p>
<p>第三方解析器需要用户自己安装，本文中使用的是<code>lxml</code>解析器，安装命令如下（它还需要先安装C语言库）。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install lxml</span><br></pre></td></tr></table></figure>
<p>下面以一个例子演示使用BeautifulSoup的基本方式，如果还想了解更多可以去参考<a target="_blank" rel="noopener" href="https://www.crummy.com/software/BeautifulSoup/bs4/doc/">BeautifulSoup文档</a>。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"></span><br><span class="line">html = <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">&lt;html&gt;&lt;head&gt;&lt;title&gt;The Dormouse&#x27;s story&lt;/title&gt;&lt;/head&gt;</span></span><br><span class="line"><span class="string">&lt;body&gt;</span></span><br><span class="line"><span class="string">&lt;p class=&quot;title&quot; name=&quot;dromouse&quot;&gt;&lt;b&gt;The Dormouse&#x27;s story&lt;/b&gt;&lt;/p&gt;</span></span><br><span class="line"><span class="string">&lt;p class=&quot;story&quot;&gt;Once upon a time there were three little sisters; and their names were</span></span><br><span class="line"><span class="string">&lt;a href=&quot;http://example.com/elsie&quot; class=&quot;sister&quot; id=&quot;link1&quot;&gt;&lt;!-- Elsie --&gt;&lt;/a&gt;,</span></span><br><span class="line"><span class="string">&lt;a href=&quot;http://example.com/lacie&quot; class=&quot;sister&quot; id=&quot;link2&quot;&gt;Lacie&lt;/a&gt; and</span></span><br><span class="line"><span class="string">&lt;a href=&quot;http://example.com/tillie&quot; class=&quot;sister&quot; id=&quot;link3&quot;&gt;Tillie&lt;/a&gt;;</span></span><br><span class="line"><span class="string">and they lived at the bottom of a well.&lt;/p&gt;</span></span><br><span class="line"><span class="string">&lt;p class=&quot;story&quot;&gt;...&lt;/p&gt;</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line">soup = BeautifulSoup(html,<span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line"><span class="comment"># 格式化输出soup中的内容</span></span><br><span class="line">print(soup.prettify())</span><br><span class="line"></span><br><span class="line"><span class="comment"># 可以通过.操作符来访问标签对象</span></span><br><span class="line">title = soup.title</span><br><span class="line">print(title)</span><br><span class="line">p = soup.p</span><br><span class="line">print(p)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获得title标签中的文本内容,这2个方法得到的结果是一样的</span></span><br><span class="line">print(title.text)</span><br><span class="line">print(title.get_text())</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获得head标签的所有子节点,contents返回的是一个列表,children返回的是一个迭代器</span></span><br><span class="line">head = soup.head</span><br><span class="line">print(head.contents)</span><br><span class="line">print(head.children)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获得所有a标签,并输出每个a标签href属性中的内容</span></span><br><span class="line">a_tags = soup.find_all(<span class="string">&#x27;a&#x27;</span>)</span><br><span class="line"><span class="keyword">for</span> a_tag <span class="keyword">in</span> a_tags:</span><br><span class="line">    print(a_tag[<span class="string">&#x27;href&#x27;</span>])</span><br><span class="line"><span class="comment"># find函数与find_all一样,只不过返回的是找到的第一个标签    </span></span><br><span class="line">print(soup.find(<span class="string">&#x27;a&#x27;</span>)[<span class="string">&#x27;href&#x27;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据属性查找,这2个方法得到的结果是一样的</span></span><br><span class="line">print(soup.find(<span class="string">&#x27;p&#x27;</span>,class_=<span class="string">&#x27;title&#x27;</span>))</span><br><span class="line">print(soup.find(<span class="string">&#x27;p&#x27;</span>,attrs=&#123;<span class="string">&#x27;class&#x27;</span>: <span class="string">&#x27;title&#x27;</span>&#125;))</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="Scrapy"><a href="#Scrapy" class="headerlink" title="Scrapy"></a>Scrapy</h3><hr>
<p><code>Scrapy</code>是一个功能强大的爬虫框架，它已经实现了一个性能高效的爬虫结构，并提供了很多供程序员自定义的配置。使用<code>Scrapy</code>只需要在它的规则上编写我们的爬虫逻辑即可。</p>
<p>首先需要先安装<code>Scrapy</code>,执行命令<code>pip install scrapy</code>。然后再执行命令<code>scrapy startproject 你的项目名</code>来生成<code>Scrapy</code>的基本项目文件夹。生成的项目结构如下。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">你的项目名&#x2F;</span><br><span class="line">    scrapy.cfg</span><br><span class="line">    你的项目名&#x2F;</span><br><span class="line">        __init__.py</span><br><span class="line">        items.py</span><br><span class="line">        pipelines.py</span><br><span class="line">        settings.py</span><br><span class="line">        spiders&#x2F;</span><br><span class="line">            __init__.py</span><br><span class="line">            ...</span><br></pre></td></tr></table></figure>
<ul>
<li><p><code>scrapy.cfg</code> : 项目的配置文件。</p>
</li>
<li><p><code>items.py</code>：物品模块，用户需要在这个模块中定义数据封装的实体类。</p>
</li>
<li><p><code>pipelines.py</code>：管道模块，用户需要在这个模块中定义处理数据的逻辑（如存储到数据库等）。</p>
</li>
<li><p><code>settings.py</code>：这个模块定义了整个项目中的各种配置变量。</p>
</li>
<li><p><code>spiders/</code>：在这个包中定义用户自己的爬虫模块。</p>
</li>
</ul>
<p>启动<code>Scrapy</code>的爬虫也很简单，只需要执行命令<code>scrapy crawl 你的爬虫名</code>。下面介绍<code>Scrapy</code>中的关键模块的演示案例，如果想要了解有关<code>Scrapy</code>的更多信息，请参考<a target="_blank" rel="noopener" href="https://doc.scrapy.org/en/0.24/intro/tutorial.html">Scrapy官方文档</a>。</p>
<h4 id="items"><a href="#items" class="headerlink" title="items"></a>items</h4><hr>
<p><strong><code>items</code>模块主要是为了将爬取到的非结构化数据封装到一个结构化对象中，自定义的<code>item</code>类必须继承自<code>scrapy.Item</code>，且每个属性都要赋值为<code>scrapy.Field()</code>。</strong></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Product</span>(<span class="params">scrapy.Item</span>):</span></span><br><span class="line">    name = scrapy.Field()</span><br><span class="line">    price = scrapy.Field()</span><br><span class="line">    stock = scrapy.Field()</span><br></pre></td></tr></table></figure>
<p>操作<code>item</code>对象就像操作一个<code>dict</code>对象一样简单。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">product &#x3D; Product()</span><br><span class="line"># 对属性赋值</span><br><span class="line">product[&#39;name&#39;] &#x3D; &#39;Sylvanas&#39;</span><br><span class="line">product[&#39;price&#39;] &#x3D; 998</span><br><span class="line"># 获得属性</span><br><span class="line">print(product[&#39;name&#39;])</span><br><span class="line">print(product[&#39;price&#39;])</span><br></pre></td></tr></table></figure>
<h4 id="pipelines"><a href="#pipelines" class="headerlink" title="pipelines"></a>pipelines</h4><hr>
<p><strong>当一个<code>Item</code>经由爬虫封装之后将会到达<code>Pipeline</code>类，你可以定义自己的<code>Pipeline</code>类来决定将<code>Item</code>的处理策略。</strong></p>
<p>每个<code>Pipeline</code>可以实现以下函数。</p>
<ul>
<li><p><code>process_item(item, spider)</code>： 每个<code>Pipeline</code>都会调用此函数来处理<code>Item</code>，这个函数必须返回一个<code>Item</code>，如果在处理过程中遇见错误，可以抛出<code>DropItem</code>异常。</p>
</li>
<li><p><code>open_spider(spider)</code>： 当<code>spider</code>开始时将会调用此函数，可以利用这个函数进行打开文件等操作。</p>
</li>
<li><p><code>close_spider(spider)</code>：当<code>spider</code>关闭时将会调用此函数，可以利用这个函数对<code>IO</code>资源进行关闭。</p>
</li>
<li><p><code>from_crawler(cls, crawler)</code>： 这个函数用于获取<code>settings.py</code>模块中的属性。注意这个函数是一个类方法。</p>
</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> scrapy.exceptions <span class="keyword">import</span> DropItem</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PricePipeline</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line"></span><br><span class="line">    vat_factor = <span class="number">1.15</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, HELLO</span>):</span></span><br><span class="line">		self.HELLO = HELLO</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span>(<span class="params">self, item, spider</span>):</span></span><br><span class="line">        <span class="keyword">if</span> item[<span class="string">&#x27;price&#x27;</span>]:</span><br><span class="line">            <span class="keyword">if</span> item[<span class="string">&#x27;price_excludes_vat&#x27;</span>]:</span><br><span class="line">                item[<span class="string">&#x27;price&#x27;</span>] = item[<span class="string">&#x27;price&#x27;</span>] * self.vat_factor</span><br><span class="line">            <span class="keyword">return</span> item</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">raise</span> DropItem(<span class="string">&quot;Missing price in %s&quot;</span> % item)</span><br><span class="line">			</span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        settings = crawler.settings <span class="comment"># 从crawler中获得settings</span></span><br><span class="line">        <span class="keyword">return</span> cls(settings[<span class="string">&#x27;HELLO&#x27;</span>]) <span class="comment"># 返回settings中的属性，将由__init__函数接收			 </span></span><br></pre></td></tr></table></figure>
<p>当定义完你的<code>Pipeline</code>后，还需要在<code>settings.py</code>中对你的<code>Pipeline</code>进行设置。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">	<span class="comment"># 后面跟的数字是优先级别</span></span><br><span class="line">    <span class="string">&#x27;pipeline类的全路径&#x27;</span>: <span class="number">300</span>,</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h4 id="spiders"><a href="#spiders" class="headerlink" title="spiders"></a>spiders</h4><hr>
<p>在<code>spiders</code>模块中，用户可以通过自定义<code>Spider</code>类来制定自己的爬虫逻辑与数据封装策略。**每个<code>Spider</code>都必须继承自<code>class scrapy.spider.Spider</code>**，这是<code>Scrapy</code>中最简单的爬虫基类，它没有什么特殊功能，<code>Scrapy</code>也提供了其他功能不同的<code>Spider</code>类供用户选择，这里就不多叙述了，可以去参考官方文档。</p>
<p>用户可以通过以下属性来自定义配置<code>Spider</code>:</p>
<ul>
<li><p><code>name</code>： 这是<code>Spider</code>的名称，<code>Scrapy</code>需要通过这个属性来定位<code>Spider</code>并启动爬虫，它是唯一且必需的。</p>
</li>
<li><p><code>allowed_domains</code>： 这个属性规定了<code>Spider</code>允许爬取的域名。</p>
</li>
<li><p><code>start_urls</code>： <code>Spider</code>开始时将抓取的网页列表。</p>
</li>
<li><p><code>start_requests()</code>： 该函数是<code>Spider</code>开始抓取时启动的函数，它只会被调用一次，有的网站必须要求用户登录，可以使用这个函数先进行模拟登录。</p>
</li>
<li><p><code>make_requests_from_url(url)</code>： 该函数接收一个<code>url</code>并返回<code>Request</code>对象。除非重写该函数，否则它会默认以<code>parse(response)</code>函数作为回调函数，并启用<code>dont_filter</code>参数（这个参数是用于过滤重复<code>url</code>的）。</p>
</li>
<li><p><code>parse(response)</code>： 当请求没有设置回调函数时，则会默认调用<code>parse(response)</code>。</p>
</li>
<li><p><code>log(message[, level, component])</code>： 用于记录日志。</p>
</li>
<li><p><code>closed(reason)</code>： 当<code>Spider</code>关闭时调用。</p>
</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MySpider</span>(<span class="params">scrapy.Spider</span>):</span></span><br><span class="line">    name = <span class="string">&#x27;example.com&#x27;</span></span><br><span class="line">    allowed_domains = [<span class="string">&#x27;example.com&#x27;</span>]</span><br><span class="line">    start_urls = [</span><br><span class="line">        <span class="string">&#x27;http://www.example.com/1.html&#x27;</span>,</span><br><span class="line">        <span class="string">&#x27;http://www.example.com/2.html&#x27;</span>,</span><br><span class="line">        <span class="string">&#x27;http://www.example.com/3.html&#x27;</span>,</span><br><span class="line">    ]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span>(<span class="params">self, response</span>):</span></span><br><span class="line">        self.log(<span class="string">&#x27;A response from %s just arrived!&#x27;</span> % response.url)</span><br></pre></td></tr></table></figure>

<h3 id="其他依赖库"><a href="#其他依赖库" class="headerlink" title="其他依赖库"></a>其他依赖库</h3><hr>
<h4 id="Requests"><a href="#Requests" class="headerlink" title="Requests"></a>Requests</h4><hr>
<p><code>Requests</code>也是一个第三方<code>python</code>库，它比<code>python</code>内置的<code>urllib</code>更加简单好用。只需要安装（<code>pip install requests</code>），然后导包后，即可轻松对网站发起请求。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="comment"># 支持http的各种类型请求</span></span><br><span class="line">r = requests.post(<span class="string">&quot;http://httpbin.org/post&quot;</span>)</span><br><span class="line">r = requests.put(<span class="string">&quot;http://httpbin.org/put&quot;</span>)</span><br><span class="line">r = requests.delete(<span class="string">&quot;http://httpbin.org/delete&quot;</span>)</span><br><span class="line">r = requests.head(<span class="string">&quot;http://httpbin.org/get&quot;</span>)</span><br><span class="line">r = requests.options(<span class="string">&quot;http://httpbin.org/get&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获得响应内容</span></span><br><span class="line">r.text <span class="comment"># 返回文本</span></span><br><span class="line">r.content <span class="comment"># 返回字节</span></span><br><span class="line">r.raw <span class="comment"># 返回原始内容</span></span><br><span class="line">r.json() <span class="comment"># 返回json</span></span><br></pre></td></tr></table></figure>
<p>关于更多的参数与内容请参考<a target="_blank" rel="noopener" href="http://docs.python-requests.org/zh_CN/latest/user/quickstart.html">Requests文档</a>。</p>
<h4 id="BloomFilter"><a href="#BloomFilter" class="headerlink" title="BloomFilter"></a>BloomFilter</h4><hr>
<p><code>BloomFilter</code>是一个用于过滤重复数据的数据结构，我们可以使用它来对重复的<code>url</code>进行过滤。本文使用的<code>BloomFilter</code>来自于<a target="_blank" rel="noopener" href="https://github.com/jaybaird/python-bloomfilter">python-bloomfilter</a>，其他操作系统用户请使用<code>pip install pybloom</code>命令安装，windows用户请使用<code>pip install pybloom-live</code>（原版对windows不友好）。</p>
<h3 id="分析"><a href="#分析" class="headerlink" title="分析"></a>分析</h3><hr>
<p>介绍了需要的依赖库之后，我们终于可以开始实现自己的图片爬虫了。我们的目标是爬<code>https://www.deviantart.com/</code>网站中的图片，在写爬虫程序之前，还需要先分析一下页面的<code>HTML</code>结构，这样才能针对性地找到图片的源地址。</p>
<p>为了保证爬到的图片的质量，我决定从热门页面开始爬，链接为<code>https://www.deviantart.com/whats-hot/</code>。</p>
<p>打开浏览器的开发者工具后，可以发现每个图片都是由一个<code>a</code>标签组成，每个<code>a</code>标签的<code>class</code>为<code>torpedo-thumb-link</code>，而这个<code>a</code>标签的<code>href</code>正好就是这张图片的详情页面（如果我们从这里就开始爬图片的话，那么爬到的可都只是缩略图）。</p>
<p><img src="http://wx1.sinaimg.cn/large/63503acbly1fjrjc53yfgj20ze0hj4fa.jpg"></p>
<p>进入到详情页后，不要马上爬取当前图片的源地址，因为当前页显示的图片并不是原始格式，我们对图片双击放大之后再使用开发者工具抓到这个图片所在的<code>img</code>标签后，再让爬虫获取这个标签中的源地址。</p>
<p><img src="http://wx3.sinaimg.cn/large/63503acbly1fjrjc67dulj21eg0h2qpr.jpg"></p>
<p>在获得图片的源地址之后，我的策略是让爬虫继续爬取该页中推荐的更多图片，通过开发者工具，可以发现这些图片都被封装在一个<code>class</code>为<code>tt-crop thumb</code>的<code>div</code>标签中，而该标签里的第一个<code>a</code>子标签正好就是这个图片的详情页链接。</p>
<p><img src="http://wx1.sinaimg.cn/large/63503acbly1fjrjc5msszj20sn0fkn80.jpg"></p>
<h3 id="初始配置"><a href="#初始配置" class="headerlink" title="初始配置"></a>初始配置</h3><hr>
<p>在对网页的<code>HTML</code>进行分析之后，可以开始写程序了，首先先用<code>Scrapy</code>的命令来初始化项目。之后在<code>settings.py</code>中做如下配置。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 这个是网络爬虫协议，爬虫访问网站时都会检查是否有robots.txt文件，</span></span><br><span class="line"><span class="comment"># 然后根据文件中的内容选择性地进行爬取，我们这里设置为False即不检查robots.txt</span></span><br><span class="line">ROBOTSTXT_OBEY = <span class="literal">False</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 图片下载的根目录路径</span></span><br><span class="line">IMAGES_STORE = <span class="string">&#x27;.&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 图片最大下载数量，当下载的图片达到这个数字时，将会手动关闭爬虫</span></span><br><span class="line">MAXIMUM_IMAGE_NUMBER = <span class="number">10000</span></span><br></pre></td></tr></table></figure>
<p>然后定义我们的<code>Item</code>。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">DeviantArtSpiderItem</span>(<span class="params">scrapy.Item</span>):</span></span><br><span class="line">    author = scrapy.Field() <span class="comment"># 作者名</span></span><br><span class="line">    image_name = scrapy.Field() <span class="comment"># 图片名</span></span><br><span class="line">    image_id = scrapy.Field() <span class="comment"># 图片id</span></span><br><span class="line">    image_src = scrapy.Field() <span class="comment"># 图片的源地址</span></span><br></pre></td></tr></table></figure>
<p>创建自己的<code>spider</code>模块与<code>Spider</code>类。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="comment"># this import package is right,if PyCharm give out warning please ignore</span></span><br><span class="line"><span class="keyword">from</span> deviant_art_spider.items <span class="keyword">import</span> DeviantArtSpiderItem</span><br><span class="line"><span class="keyword">from</span> pybloom_live <span class="keyword">import</span> BloomFilter</span><br><span class="line"><span class="keyword">from</span> scrapy.contrib.linkextractors.lxmlhtml <span class="keyword">import</span> LxmlLinkExtractor</span><br><span class="line"><span class="keyword">from</span> scrapy.contrib.spiders <span class="keyword">import</span> CrawlSpider, Rule</span><br><span class="line"><span class="keyword">from</span> scrapy.http <span class="keyword">import</span> Request</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">DeviantArtImageSpider</span>(<span class="params">CrawlSpider</span>):</span></span><br><span class="line">    name = <span class="string">&#x27;deviant_art_image_spider&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 我不想让scrapy帮助过滤所以设置为空</span></span><br><span class="line">    allowed_domains = <span class="string">&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    start_urls = [<span class="string">&#x27;https://www.deviantart.com/whats-hot/&#x27;</span>]</span><br><span class="line"></span><br><span class="line">    rules = (</span><br><span class="line">        Rule(LxmlLinkExtractor(</span><br><span class="line">            allow=&#123;<span class="string">&#x27;https://www.deviantart.com/whats-hot/[\?\w+=\d+]*&#x27;</span>, &#125;),</span><br><span class="line">            callback=<span class="string">&#x27;parse_page&#x27;</span>, <span class="comment"># 设置回调函数</span></span><br><span class="line">            follow=<span class="literal">True</span> <span class="comment"># 允许爬虫不断地跟随链接进行爬取</span></span><br><span class="line">        ),</span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36&quot;</span></span><br><span class="line">                      <span class="string">&quot; (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36&quot;</span>,</span><br><span class="line">        <span class="string">&quot;Referer&quot;</span>: <span class="string">&quot;https://www.deviantart.com/&quot;</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">	<span class="comment"># 初始化BloomFilter</span></span><br><span class="line">    <span class="built_in">filter</span> = BloomFilter(capacity=<span class="number">15000</span>)</span><br></pre></td></tr></table></figure>
<p><code>DeviantArtImageSpider</code>继承自<code>CrawlSpider</code>，该类是<code>Scrapy</code>最常用的<code>Spider</code>类，它通过<code>Rule</code>类来定义爬取链接的规则，上述代码中使用了正则表达式<code>https://www.deviantart.com/whats-hot/[\?\w+=\d+]*</code>，这个正则表达式将访问每一页的热门页面。</p>
<h3 id="解析热门页面"><a href="#解析热门页面" class="headerlink" title="解析热门页面"></a>解析热门页面</h3><hr>
<p>爬虫启动时将会先访问热门页面，请求得到响应之后会调用回调函数，我们需要在这个回调函数中获取上述分析中得到的<code>&lt;a class = &#39;torpedo-thumb-link&#39;&gt;</code>标签，然后抽取出每张图片的详情页链接。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">parse_page</span>(<span class="params">self, response</span>):</span></span><br><span class="line">       soup = self._init_soup(response, <span class="string">&#x27;[PREPARING PARSE PAGE]&#x27;</span>)</span><br><span class="line">       <span class="keyword">if</span> soup <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line">	<span class="comment"># 找到所有class为torpedo-thumb-link的a标签	</span></span><br><span class="line">       all_a_tag = soup.find_all(<span class="string">&#x27;a&#x27;</span>, class_=<span class="string">&#x27;torpedo-thumb-link&#x27;</span>)</span><br><span class="line">       <span class="keyword">if</span> all_a_tag <span class="keyword">is</span> <span class="keyword">not</span> <span class="literal">None</span> <span class="keyword">and</span> <span class="built_in">len</span>(all_a_tag) &gt; <span class="number">0</span>:</span><br><span class="line">           <span class="keyword">for</span> a_tag <span class="keyword">in</span> all_a_tag:</span><br><span class="line">			<span class="comment"># 提取图片详情页，然后对详情页链接发起请求，并设置回调函数</span></span><br><span class="line">               detail_link = a_tag[<span class="string">&#x27;href&#x27;</span>]</span><br><span class="line">               request = Request(</span><br><span class="line">                   url=detail_link,</span><br><span class="line">                   headers=self.headers,</span><br><span class="line">                   callback=self.parse_detail_page</span><br><span class="line">               )</span><br><span class="line">			<span class="comment"># 通过request与response对象来传递Item</span></span><br><span class="line">               request.meta[<span class="string">&#x27;item&#x27;</span>] = DeviantArtSpiderItem()</span><br><span class="line">               <span class="keyword">yield</span> request</span><br><span class="line">       <span class="keyword">else</span>:</span><br><span class="line">           self.logger.debug(<span class="string">&#x27;[PARSE FAILED] get &lt;a&gt; tag failed&#x27;</span>)</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 初始化BeautifulSoup对象</span></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">_init_soup</span>(<span class="params">self, response, log</span>):</span></span><br><span class="line">       url = response.url</span><br><span class="line">       self.headers[<span class="string">&#x27;Referer&#x27;</span>] = url</span><br><span class="line">       self.logger.debug(log + <span class="string">&#x27; &#x27;</span> + url)</span><br><span class="line">       body = requests.get(url, headers=self.headers, timeout=<span class="number">2</span>).content</span><br><span class="line">       soup = BeautifulSoup(body, <span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line">       <span class="keyword">if</span> soup <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">           self.logger.debug(<span class="string">&#x27;[PARSE FAILED] read %s body failed&#x27;</span> % url)</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line">       <span class="keyword">return</span> soup</span><br></pre></td></tr></table></figure>


<h3 id="解析详情页"><a href="#解析详情页" class="headerlink" title="解析详情页"></a>解析详情页</h3><hr>
<p><code>parse_page()</code>函数会不断地发送请求到详情页链接，解析详情页的回调函数需要处理数据封装到<code>Item</code>，还需要提取详情页中更多图片的详情链接然后发送请求。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">parse_detail_page</span>(<span class="params">self, response</span>):</span></span><br><span class="line">       <span class="keyword">if</span> response.url <span class="keyword">in</span> self.<span class="built_in">filter</span>:</span><br><span class="line">           self.logger.debug(<span class="string">&#x27;[REPETITION] already parse url %s &#x27;</span> % response.url)</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line">       soup = self._init_soup(response, <span class="string">&#x27;[PREPARING DETAIL PAGE]&#x27;</span>)</span><br><span class="line">       <span class="keyword">if</span> soup <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line">	<span class="comment"># 包装Item并返回	</span></span><br><span class="line">       <span class="keyword">yield</span> self.packing_item(response.meta[<span class="string">&#x27;item&#x27;</span>], soup)</span><br><span class="line">       self.<span class="built_in">filter</span>.add(response.url)</span><br><span class="line">       <span class="comment"># 继续抓取当前页中的其他图片</span></span><br><span class="line">       all_div_tag = soup.find_all(<span class="string">&#x27;div&#x27;</span>, class_=<span class="string">&#x27;tt-crop thumb&#x27;</span>)</span><br><span class="line">       <span class="keyword">if</span> all_div_tag <span class="keyword">is</span> <span class="keyword">not</span> <span class="literal">None</span> <span class="keyword">and</span> <span class="built_in">len</span>(all_div_tag) &gt; <span class="number">0</span>:</span><br><span class="line">           <span class="keyword">for</span> div_tag <span class="keyword">in</span> all_div_tag:</span><br><span class="line">               detail_link = div_tag.find(<span class="string">&#x27;a&#x27;</span>)[<span class="string">&#x27;href&#x27;</span>]</span><br><span class="line">               request = Request(</span><br><span class="line">                   url=detail_link,</span><br><span class="line">                   headers=self.headers,</span><br><span class="line">                   callback=self.parse_detail_page</span><br><span class="line">               )</span><br><span class="line">               request.meta[<span class="string">&#x27;item&#x27;</span>] = DeviantArtSpiderItem()</span><br><span class="line">               <span class="keyword">yield</span> request</span><br><span class="line">       <span class="keyword">else</span>:</span><br><span class="line">           self.logger.debug(<span class="string">&#x27;[PARSE FAILED] get &lt;div&gt; tag failed&#x27;</span>)</span><br><span class="line">           <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 封装数据到Item</span></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">packing_item</span>(<span class="params">self, item, soup</span>):</span></span><br><span class="line">       self.logger.debug(<span class="string">&#x27;[PREPARING PACKING ITEM]..........&#x27;</span>)</span><br><span class="line">       img = soup.find(<span class="string">&#x27;img&#x27;</span>, class_=<span class="string">&#x27;dev-content-full&#x27;</span>)</span><br><span class="line">       img_alt = img[<span class="string">&#x27;alt&#x27;</span>] <span class="comment"># alt属性中保存了图片名与作者名</span></span><br><span class="line">       item[<span class="string">&#x27;image_name&#x27;</span>] = img_alt[:img_alt.find(<span class="string">&#x27;by&#x27;</span>) - <span class="number">1</span>]</span><br><span class="line">       item[<span class="string">&#x27;author&#x27;</span>] = img_alt[img_alt.find(<span class="string">&#x27;by&#x27;</span>) + <span class="number">2</span>:]</span><br><span class="line">       item[<span class="string">&#x27;image_id&#x27;</span>] = img[<span class="string">&#x27;data-embed-id&#x27;</span>] <span class="comment"># data-embed-id属性保存了图片id</span></span><br><span class="line">       item[<span class="string">&#x27;image_src&#x27;</span>] = img[<span class="string">&#x27;src&#x27;</span>]</span><br><span class="line">       self.logger.debug(<span class="string">&#x27;[PACKING ITEM FINISHED] %s &#x27;</span> % item)</span><br><span class="line">       <span class="keyword">return</span> item</span><br></pre></td></tr></table></figure>
<h3 id="处理Item"><a href="#处理Item" class="headerlink" title="处理Item"></a>处理Item</h3><hr>
<p>对于<code>Item</code>的处理，只是简单地将图片命名与下载到本地。我没有使用多进程或者多线程，也没有使用<code>Scrapy</code>自带的<code>ImagePipeline</code>（自由度不高），有兴趣的童鞋可以自己选择实现。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> threading</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> scrapy.exceptions <span class="keyword">import</span> DropItem, CloseSpider</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">DeviantArtSpiderPipeline</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, IMAGE_STORE, MAXIMUM_IMAGE_NUMBER</span>):</span></span><br><span class="line">        <span class="keyword">if</span> IMAGE_STORE <span class="keyword">is</span> <span class="literal">None</span> <span class="keyword">or</span> MAXIMUM_IMAGE_NUMBER <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="keyword">raise</span> CloseSpider(<span class="string">&#x27;Pipeline load settings failed&#x27;</span>)</span><br><span class="line">        self.IMAGE_STORE = IMAGE_STORE</span><br><span class="line">        self.MAXIMUM_IMAGE_NUMBER = MAXIMUM_IMAGE_NUMBER</span><br><span class="line">        <span class="comment"># 记录当前下载的图片数量</span></span><br><span class="line">        self.image_max_counter = <span class="number">0</span></span><br><span class="line">        <span class="comment"># 根据图片数量创建文件夹，每1000张在一个文件夹中</span></span><br><span class="line">        self.dir_counter = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span>(<span class="params">self, item, spider</span>):</span></span><br><span class="line">        <span class="keyword">if</span> item <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="keyword">raise</span> DropItem(<span class="string">&#x27;Item is null&#x27;</span>)</span><br><span class="line">        dir_path = self.make_dir()</span><br><span class="line">		<span class="comment"># 拼接图片名称</span></span><br><span class="line">        image_final_name = item[<span class="string">&#x27;image_name&#x27;</span>] + <span class="string">&#x27;-&#x27;</span> + item[<span class="string">&#x27;image_id&#x27;</span>] + <span class="string">&#x27;-by@&#x27;</span> + item[<span class="string">&#x27;author&#x27;</span>] + <span class="string">&#x27;.jpg&#x27;</span></span><br><span class="line">        dest_path = os.path.join(dir_path, image_final_name)</span><br><span class="line">        self.download_image(item[<span class="string">&#x27;image_src&#x27;</span>], dest_path)</span><br><span class="line">        self.image_max_counter += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> self.image_max_counter &gt;= self.MAXIMUM_IMAGE_NUMBER:</span><br><span class="line">            <span class="keyword">raise</span> CloseSpider(<span class="string">&#x27;Current downloaded image already equal maximum number&#x27;</span>)</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">make_dir</span>(<span class="params">self</span>):</span></span><br><span class="line">        print(<span class="string">&#x27;[IMAGE_CURRENT NUMBER] %d &#x27;</span> % self.image_max_counter)</span><br><span class="line">        <span class="keyword">if</span> self.image_max_counter % <span class="number">1000</span> == <span class="number">0</span>:</span><br><span class="line">            self.dir_counter += <span class="number">1</span></span><br><span class="line">        path = os.path.abspath(self.IMAGE_STORE)</span><br><span class="line">        path = os.path.join(path, <span class="string">&#x27;crawl_images&#x27;</span>)</span><br><span class="line">        path = os.path.join(path, <span class="string">&#x27;dir-&#x27;</span> + <span class="built_in">str</span>(self.dir_counter))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(path):</span><br><span class="line">            os.makedirs(path)</span><br><span class="line">            print(<span class="string">&#x27;[CREATED DIR] %s &#x27;</span> % path)</span><br><span class="line">        <span class="keyword">return</span> path</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">download_image</span>(<span class="params">self, src, dest</span>):</span></span><br><span class="line">        print(<span class="string">&#x27;[Thread %s] preparing download image.....&#x27;</span> % threading.current_thread().name)</span><br><span class="line">        response = requests.get(src, timeout=<span class="number">2</span>)</span><br><span class="line">        <span class="keyword">if</span> response.status_code == <span class="number">200</span>:</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(dest, <span class="string">&#x27;wb&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(response.content)</span><br><span class="line">                print(<span class="string">&#x27;[DOWNLOAD FINISHED] from %s to %s &#x27;</span> % (src, dest))</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">raise</span> DropItem(<span class="string">&#x27;[Thread %s] request image src failed status code = %s&#x27;</span></span><br><span class="line">                           % (threading.current_thread().name, response.status_code))</span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        settings = crawler.settings</span><br><span class="line">        <span class="keyword">return</span> cls(settings[<span class="string">&#x27;IMAGES_STORE&#x27;</span>], settings[<span class="string">&#x27;MAXIMUM_IMAGE_NUMBER&#x27;</span>])</span><br></pre></td></tr></table></figure>
<p>在<code>settings.py</code>中注册该<code>Pipeline</code></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">    <span class="string">&#x27;deviant_art_spider.pipelines.DeviantArtSpiderPipeline&#x27;</span>: <span class="number">300</span>,</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="IP代理池"><a href="#IP代理池" class="headerlink" title="IP代理池"></a>IP代理池</h3><hr>
<p>有些网站会有反爬虫机制，为了解决这个问题，每次请求都使用不同的<code>IP</code>代理，有很多网站提供<code>IP</code>代理服务，我们需要写一个爬虫从<a target="_blank" rel="noopener" href="http://www.ip3366.net/free/">云代理</a>中抓取它提供的免费<code>IP</code>代理（免费<code>IP</code>很不稳定，而且我用了代理之后反而各种请求失败了Orz…）。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ProxiesSpider</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, max_page_number=<span class="number">10</span></span>):</span></span><br><span class="line">        self.seed = <span class="string">&#x27;http://www.ip3366.net/free/&#x27;</span></span><br><span class="line">        self.max_page_number = max_page_number <span class="comment"># 最大页数</span></span><br><span class="line">        self.crawled_proxies = [] <span class="comment"># 爬到的ip,每个元素都是一个dict</span></span><br><span class="line">        self.verified_proxies = [] <span class="comment"># 校验过的ip</span></span><br><span class="line">        self.headers = &#123;</span><br><span class="line">            <span class="string">&#x27;Accept&#x27;</span>: <span class="string">&#x27;*/*&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;User-Agent&#x27;</span>: <span class="string">&#x27;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)&#x27;</span></span><br><span class="line">                          <span class="string">&#x27; Chrome/45.0.2454.101 Safari/537.36&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;Accept-Language&#x27;</span>: <span class="string">&#x27;zh-CN,zh;q=0.8&#x27;</span></span><br><span class="line">        &#125;</span><br><span class="line">        self.tocrawl_url = []</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">crawl</span>(<span class="params">self</span>):</span></span><br><span class="line">        self.tocrawl_url.append(self.seed)</span><br><span class="line">        page_counter = <span class="number">1</span></span><br><span class="line">        <span class="keyword">while</span> self.tocrawl_url:</span><br><span class="line">            <span class="keyword">if</span> page_counter &gt; self.max_page_number:</span><br><span class="line">                <span class="keyword">break</span></span><br><span class="line">            url = self.tocrawl_url.pop()</span><br><span class="line">            body = requests.get(url=url, headers=self.headers, params=&#123;<span class="string">&#x27;page&#x27;</span>: page_counter&#125;).content</span><br><span class="line">            soup = BeautifulSoup(body, <span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line">            <span class="keyword">if</span> soup <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">                print(<span class="string">&#x27;PARSE PAGE FAILED.......&#x27;</span>)</span><br><span class="line">                <span class="keyword">continue</span></span><br><span class="line">            self.parse_page(soup)</span><br><span class="line">            print(<span class="string">&#x27;Parse page %s done&#x27;</span> % (url + <span class="string">&#x27;?page=&#x27;</span> + <span class="built_in">str</span>(page_counter)))</span><br><span class="line">            page_counter += <span class="number">1</span></span><br><span class="line">            self.tocrawl_url.append(url)</span><br><span class="line">        self.verify_proxies()</span><br><span class="line">        self.download()</span><br><span class="line">	</span><br><span class="line">	<span class="comment"># 解析页面并封装</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_page</span>(<span class="params">self, soup</span>):</span></span><br><span class="line">        table = soup.find(<span class="string">&#x27;table&#x27;</span>, class_=<span class="string">&#x27;table table-bordered table-striped&#x27;</span>)</span><br><span class="line">        tr_list = table.tbody.find_all(<span class="string">&#x27;tr&#x27;</span>)</span><br><span class="line">        <span class="keyword">for</span> tr <span class="keyword">in</span> tr_list:</span><br><span class="line">            ip = tr.contents[<span class="number">1</span>].text</span><br><span class="line">            port = tr.contents[<span class="number">3</span>].text</span><br><span class="line">            protocol = tr.contents[<span class="number">7</span>].text.lower()</span><br><span class="line">            url = protocol + <span class="string">&#x27;://&#x27;</span> + ip + <span class="string">&#x27;:&#x27;</span> + port</span><br><span class="line">            self.crawled_proxies.append(&#123;url: protocol&#125;)</span><br><span class="line">            print(<span class="string">&#x27;Add url %s to crawled_proxies&#x27;</span> % url)</span><br><span class="line"></span><br><span class="line">	<span class="comment"># 对ip进行校验</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">verify_proxies</span>(<span class="params">self</span>):</span></span><br><span class="line">        print(<span class="string">&#x27;Start verify proxies.......&#x27;</span>)</span><br><span class="line">        <span class="keyword">while</span> self.crawled_proxies:</span><br><span class="line">            self.verify_proxy(self.crawled_proxies.pop())</span><br><span class="line">        print(<span class="string">&#x27;Verify proxies done.....&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">verify_proxy</span>(<span class="params">self, proxy</span>):</span></span><br><span class="line">        proxies = &#123;&#125;</span><br><span class="line">        <span class="keyword">for</span> key <span class="keyword">in</span> proxy:</span><br><span class="line">            proxies[<span class="built_in">str</span>(proxy[key])] = key <span class="comment"># requests的proxies的格式必须为 协议 : 地址</span></span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="keyword">if</span> requests.get(<span class="string">&#x27;https://www.deviantart.com/&#x27;</span>, proxies=proxies, timeout=<span class="number">2</span>).status_code == <span class="number">200</span>:</span><br><span class="line">                print(<span class="string">&#x27;verify proxy success %s &#x27;</span> % proxies)</span><br><span class="line">                self.verified_proxies.append(proxy)</span><br><span class="line">        <span class="keyword">except</span>:</span><br><span class="line">            print(<span class="string">&#x27;verify proxy fail %s &#x27;</span> % proxies)</span><br><span class="line">	</span><br><span class="line">	<span class="comment"># 保存到文件中</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">download</span>(<span class="params">self</span>):</span></span><br><span class="line">        current_path = os.getcwd()</span><br><span class="line">        parent_path = os.path.dirname(current_path)</span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(parent_path + <span class="string">&#x27;\proxies.txt&#x27;</span>, <span class="string">&#x27;w&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">            <span class="keyword">for</span> proxy <span class="keyword">in</span> self.verified_proxies:</span><br><span class="line">                <span class="keyword">for</span> key <span class="keyword">in</span> proxy.keys():</span><br><span class="line">                    f.write(key + <span class="string">&#x27;\n&#x27;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    spider = ProxiesSpider()</span><br><span class="line">    spider.crawl()</span><br></pre></td></tr></table></figure>
<p>得到了<code>IP</code>代理池之后，还要在<code>Scrapy</code>的<code>middlewares.py</code>模块定义代理中间件类。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> signals</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ProxyMiddleware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">	<span class="comment"># 每次请求前从IP代理池中选择一个IP代理并进行设置</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self, request, spider</span>):</span></span><br><span class="line">        proxy = self.get_proxy(self.make_path())</span><br><span class="line">        print(<span class="string">&#x27;Acquire proxy %s &#x27;</span> % proxy)</span><br><span class="line">        request.meta[<span class="string">&#x27;proxy&#x27;</span>] = proxy</span><br><span class="line">	</span><br><span class="line">	<span class="comment"># 请求失败，重新设置IP代理</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span>(<span class="params">self, request, response, spider</span>):</span></span><br><span class="line">        <span class="keyword">if</span> response.status != <span class="number">200</span>:</span><br><span class="line">            proxy = self.get_proxy(self.make_path())</span><br><span class="line">            print(<span class="string">&#x27;Response status code is not 200,try reset request proxy %s &#x27;</span> % proxy)</span><br><span class="line">            request.meta[<span class="string">&#x27;proxy&#x27;</span>] = proxy</span><br><span class="line">            <span class="keyword">return</span> request</span><br><span class="line">        <span class="keyword">return</span> response</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">make_path</span>(<span class="params">self</span>):</span></span><br><span class="line">        current = os.path.abspath(<span class="string">&#x27;.&#x27;</span>)</span><br><span class="line">        parent = os.path.dirname(current)</span><br><span class="line">        <span class="keyword">return</span> os.path.dirname(parent) + <span class="string">&#x27;\proxies.txt&#x27;</span></span><br><span class="line">	</span><br><span class="line">	<span class="comment"># 从IP代理文件中随机获得一个IP代理地址</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_proxy</span>(<span class="params">self, path</span>):</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.isfile(path):</span><br><span class="line">            print(<span class="string">&#x27;[LOADING PROXY] loading proxies failed proxies file is not exist&#x27;</span>)</span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(path, <span class="string">&#x27;r&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                proxies = f.readlines()</span><br><span class="line">            <span class="keyword">if</span> proxies:</span><br><span class="line">                <span class="keyword">break</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                time.sleep(<span class="number">1</span>)</span><br><span class="line">        <span class="keyword">return</span> random.choice(proxies).strip()</span><br></pre></td></tr></table></figure>
<p>最后在<code>settings.py</code>中进行注册。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">DOWNLOADER_MIDDLEWARES = &#123;</span><br><span class="line">	<span class="comment"># 这个中间件是由scrapy提供的，并且它是必需的</span></span><br><span class="line">    <span class="string">&#x27;scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware&#x27;</span>: <span class="number">543</span>, </span><br><span class="line">	<span class="comment"># 我们自定义的代理中间件</span></span><br><span class="line">    <span class="string">&#x27;deviant_art_spider.middlewares.ProxyMiddleware&#x27;</span>: <span class="number">540</span> </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="End"><a href="#End" class="headerlink" title="End"></a>End</h3><hr>
<p>我们的图片爬虫已经完成了，执行命令<code>scrapy crawl deviant_art_image_spider</code>，然后尽情搜集图片吧！</p>
<p><img src="http://wx2.sinaimg.cn/large/63503acbly1fjrmgmpigfg214o0ihe2b.gif"></p>
<p><strong><a target="_blank" rel="noopener" href="https://github.com/SylvanasSun/scrapy-picture-spider">想要获得本文中的完整源代码与P站爬虫请点我，顺便求个star…</a></strong></p>
<blockquote>
<p>最近心血来潮想要写爬虫，所以花了点时间过了一遍<code>python</code>语法便匆匆上手了，代码写的有点丑也不够pythonic，各位看官求请吐槽。</p>
</blockquote>

    </div>

    
    
    

    <footer class="post-footer">
          <div class="post-tags">
              <a href="/yuwanzi.io/tags/2017/" rel="tag"># 2017</a>
              <a href="/yuwanzi.io/tags/python/" rel="tag"># python</a>
              <a href="/yuwanzi.io/tags/%E7%88%AC%E8%99%AB/" rel="tag"># 爬虫</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2017/09/08/2017-09-08-ComputerStructure/" rel="prev" title="探索计算机的结构与核心概念">
                  <i class="fa fa-chevron-left"></i> 探索计算机的结构与核心概念
                </a>
            </div>
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2017/10/03/2017-10-03-BrowserCriticalRenderingPath/" rel="next" title="浏览器渲染过程与性能优化">
                  浏览器渲染过程与性能优化 <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
    </footer>
  </article>
</div>







<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      const activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      const commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>
</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">玉丸子</span>
</div>
  <div class="powered-by">Erstellt mit  <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/muse/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a>
  </div>

    </div>
  </footer>

  
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
<script src="/yuwanzi.io/js/utils.js"></script><script src="/yuwanzi.io/js/motion.js"></script><script src="/yuwanzi.io/js/schemes/muse.js"></script><script src="/yuwanzi.io/js/next-boot.js"></script>

  






  





</body>
</html>
